package wiki;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;

public class NewWiki {
	
	public static final String version = "0.1.0alpha";
	
	/**
	 * The URL domain of the wiki to be used. Examples:
	 * en.wikipedia.org for the English Wikipedia
	 * commons.wikimedia.org for the Wikimedia Commons
	 */
	public final String DOMAIN;
	/**
	 * Usually /w/, this is the part that appears just after the domain
	 * in the address bar when editing a page
	 */
	public final String SCRIPTPATH;
	/**
	 * Base of the URL for viewing pages
	 */
	public final String URL_BASE;
	/**
	 * Base of the URL for querying information from the API
	 */
	public final String QUERY_BASE;
	
	/**
	 * The lowest output level, prints all messages, including entering/
	 * exiting method notices and major breakpoints in large methods
	 */
	public final static int DEBUG = 0;
	/**
	 * Second lowest output level, prints messages about routine information
	 * such as replag times, missing but unexpected attributes, etc.
	 */
	public final static int INFO = 1;
	/**
	 * Third lowest output level, prints messages that may be important to
	 * note in a log but aren't serious problems. Includes delays due to
	 * high replag and notices about successful wiki-modifying actions
	 * such as edits, moves, etc.
	 */
	public final static int NOTICE = 2;
	/**
	 * Third highest output level, prints messages that may cause problems
	 * for the program if not handled properly. Includes notices about
	 * unexpected input, missing attributes that ARE expected, etc.
	 */
	public final static int WARNING = 3;
	/**
	 * Second highest output level, prints messages about serious problems
	 * that will likely result in failure of the task but that should permit
	 * a clean shutdown. Includes API errors and exceptions caused by network
	 * errors.
	 */
	public final static int ERROR = 4;
	/**
	 * Highest output level, prints only messages regarding fatal errors that
	 * cause an immediate, improper termination of the program. Limited only
	 * to exceptions that are thrown as a result of user error.
	 */
	public final static int EMERGENCY = 5;	
	
	/**
	 * Delay between POSTs in seconds. Default is 12 (5/min)
	 */
	private int editThrottle = 12;
	/**
	 * Delay between GETs in seconds. Default is 2 (30/min)
	 */
	private int queryThrottle = 2;
	/**
	 * Delay between checking server lag in seconds. Default is 60
	 */
	private int lagCheckDelay = 60;
	private long lastPostTime = 0;
	private long lastGetTime = 0;
	private long lastLagCheck = 0;
	/**
	 * Maximum tolerated database replication lag in seconds
	 */
	private int maxLag = 10;
	/**
	 * Logged messages above this level will be printed to System.out
	 */
	private static int outputLevel = DEBUG;
	/**
	 * Logged messages above this level will be printed to System.err.
	 * Should be higher than or equal to outputLevel.
	 */
	private static int errorLevel = WARNING;
	private static Calendar time = Calendar.getInstance();
	private HashMap<String, String> cookies = new HashMap<String, String>();
	private HashMap<String, String> sessionCookies = new HashMap<String, String>();
	private final String useragent = "New Java Wiki Framework version " + version;
	
	/**
	 * Creates a new NewWiki object with the given domain and default script path.
	 * @param domain ex. "en.wikipedia.org"
	 */
	public NewWiki(String domain){
		this(domain, "/w/");
	}
	
	/**
	 * Creates a new NewWiki object with the given domain and script path.
	 * @param domain ex. "en.wikipedia.org"
	 * @param scriptpath Usually "/w/" - http://en.wikipedia.org<b>/w/</b>index.php?...
	 */
	public NewWiki(String domain, String scriptpath){
		this.DOMAIN = domain;
		this.SCRIPTPATH = scriptpath;
		URL_BASE = "http://" + DOMAIN + SCRIPTPATH + "index.php?title=";
		QUERY_BASE = "http://" + DOMAIN + SCRIPTPATH + "api.php?format=xml&action=";
		time.setTimeZone(TimeZone.getTimeZone("Universal"));
	}
	
	public APIDataList allUsers(String from, String to, String prefix, String group, String[] props, 
			int limit, boolean editsOnly){
		addToLog("Entering method", "allUsers()", DEBUG);
		String url = "query&list=allusers";

		if(notEmpty(from)){
			url += "&aufrom=" + encode(from);
		}
		if(notEmpty(to)){
			url += "&auto=" + encode(to);
		}
		if(notEmpty(prefix)){
			url += "&auprefix=" + encode(prefix);
		}
		if(notEmpty(group)){
			UserGroup groupType = UserGroup.getType(group);
			if(groupType != null){
				url += "&augroup=" + groupType.getName();
			}
		}
		if(props != null){
			url += "&auprop=";
			String proplist = "";
			for(int i = 0; i < props.length; i++){
				String prop = props[i];
				if(prop.equals("blockinfo") || prop.equals("groups") || 
						prop.equals("editcount") || prop.equals("registration")){
					if(!proplist.isEmpty()){
						proplist += "|";
					}
					proplist += prop;
				}
				else{
					addToLog("Unrecognized property " + prop + " ignored.", "allUsers()", WARNING);
				}
			}
			url += proplist;
		}
		if(limit > 0 && limit <= 5000){
			url += "&aulimit=" + limit;
		}
		if(editsOnly){
			url += "&auwitheditsonly=" + editsOnly;
		}
		addToLog("Query construction complete", "allUsers()", DEBUG);
		
		boolean dontContinue = false;
		// If we expect a finite number of users, and the user didn't say how many...
		if(((notEmpty(from) && notEmpty(to)) || notEmpty(prefix) || notEmpty(group)) &&
				limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		else if(limit <= 0){
			dontContinue = true;
		}
		
		boolean notdone = true;
		APIDataList users = new APIDataList();
		while(notdone && users.size() < limit){
			notdone = false;
			String result = doRequest(url, "allUsers()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue") && !dontContinue){
				addToLog("Collecting continuation information", "allUsers()", DEBUG);
				notdone = true;
				url = getContinuationInfo(result, url, "aufrom");
				
			}
			addToLog("Gathering user data", "allUsers()", DEBUG);
			
			result = trimResult(result, "allusers");
			if(result == null){
				addToLog("Server returned no results!", "allUsers()", WARNING);
				return null;
			}
			
			users.addAll(collectData(result, "u"));
		}
		
		addToLog("Leaving method", "allUsers()", DEBUG);
		return users;
	}

	public APIDataList allImages(String from, String to, String prefix, 
			int minsize, int maxsize, int limit, boolean descending, 
			String sha, String sha36, String[] props){
		APIDataList imagedata = new APIDataList();
		addToLog("Entering method", "allImages()", DEBUG);
		String url = "query&list=allimages";
		if(notEmpty(from)){
			url += "&aifrom=" + encode(from);
		}
		if(notEmpty(to)){
			url += "&aito=" + encode(to);
		}
		if(notEmpty(prefix)){
			url += "&aiprefix=" + encode(prefix);
		}
		if(minsize > 0){
			url += "&aiminsize=" + minsize;
		}
		if(maxsize > 0 && maxsize >= minsize){
			url += "&aimaxsize=" + maxsize;
		}
		if(limit > 0){
			url += "&ailimit=" + limit;
		}
		if(descending){
			url += "&aidir=descending";
		}
		if(notEmpty(sha) && sha.matches("[0-9a-f]{40,40}")){
			url += "&aisha1=" + sha;
		}
		else if(notEmpty(sha36) && sha36.matches("[0-9a-z]{31,31}")){
			url += "&aisha1base36=" + sha36;
		}
		if(props != null){
			url += "&aiprop=";
			String proplist = "";
			for(int i = 0; i < props.length; i++){
				String prop = props[i];
				if(prop.equals("timestamp") || prop.equals("user") || 
						prop.equals("userid") || prop.equals("comment") ||
						prop.equals("url") || prop.equals("size") ||
						prop.equals("dimensions") || prop.equals("sha1") ||
						prop.equals("mime") || prop.equals("metadata") ||
						prop.equals("archivename") || prop.equals("bitdepth")){
					if(!proplist.isEmpty()){
						proplist += "|";
					}
					proplist += prop;
				}
				else{
					addToLog("Unrecognized property " + prop + " ignored.", "allImages()", WARNING);
				}
			}
			url += proplist;
		}
		
		addToLog("Query construction complete", "allImages()", DEBUG);
		
		boolean dontContinue = false;
		// If we expect a finite number of users, and the user didn't say how many...
		if(((notEmpty(from) && notEmpty(to)) || notEmpty(prefix) || notEmpty(sha) || 
				notEmpty(sha36) || (minsize > 0 && maxsize > 0 && maxsize >= minsize)) &&
				limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		else if(limit <= 0){
			dontContinue = true;
		}
		
		boolean notdone = true;
		while(notdone && (imagedata == null || imagedata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "allImages()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue") && !dontContinue){
				addToLog("Collecting continuation information", "allImages()", DEBUG);
				notdone = true;
				url = getContinuationInfo(result, url, "aifrom");
				
			}
			addToLog("Gathering image data", "allImages()", DEBUG);
			
			result = trimResult(result, "allimages");
			if(result == null){
				addToLog("Server returned no results!", "allImages()", WARNING);
				return null;
			}
			
			imagedata.addAll(collectData(result, "img"));
			
		}
		
		return imagedata;
	}
	
	public APIDataList allPages(String from, String to, String prefix, Namespace namespace, int filterRedirs, 
			int minsize, int maxsize, String[] protTypes, String protLevel, int cascade, int limit, 
			boolean descending, int langlinks){
		APIDataList pagedata = new APIDataList();
		addToLog("Entering method", "allPages()", DEBUG);
		String url = "query&list=allpages";
		if(notEmpty(from)){
			url += "&apfrom=" + encode(from);
		}
		if(notEmpty(to)){
			url += "&apto=" + encode(to);
		}
		if(notEmpty(prefix)){
			url += "&apprefix=" + encode(prefix);
		}
		if(minsize > 0){
			url += "&apminsize=" + minsize;
		}
		if(maxsize > 0 && maxsize >= minsize){
			url += "&apmaxsize=" + maxsize;
		}
		if(limit > 0){
			url += "&aplimit=" + limit;
		}
		if(descending){
			url += "&apdir=descending";
		}
		if(namespace != null){
			url += "&apnamespace=" + namespace.getNumber();
		}
		if(filterRedirs != 0){
			url += "&apfilterredir=" + (filterRedirs > 0 ? "redirects" : "nonredirects");
		}
		if(protTypes != null){
			for(String protType : protTypes){
				if(protType.equals("edit") || protType.equals("move") || protType.equals("upload")){
					url += "&apprtype=" + protType;
				}
			}
			if(notEmpty(protLevel)){
				if(protLevel.equals("sysop") || protLevel.equals("autoconfirmed") ||
						protLevel.equals("sysop|autoconfirmed") ||
						protLevel.equals("autoconfirmed|sysop")){
					url += "&apprlevel=" + protLevel;
				}
			}
			if(cascade != 0){
				url += "&apprfiltercascade=" + (cascade > 0 ? "cascading" : "noncascading");
			}
		}
		if(langlinks != 0){
			url += "&apfilterlanglinks=" + (langlinks > 0 ? "withlanglinks" : "withoutlanglinks");
		}
		
		addToLog("Query construction complete", "allImages()", DEBUG);
		
		boolean dontContinue = false;
		// If we expect a finite number of users, and the user didn't say how many...
		if(((notEmpty(from) && notEmpty(to)) || notEmpty(prefix) || (protTypes != null && cascade > 0) || 
				(minsize > 0 && maxsize > 0 && maxsize >= minsize)) &&
				limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		else if(limit <= 0){
			dontContinue = true;
		}
		
		boolean notdone = true;
		while(notdone && (pagedata == null || pagedata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "allPages()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue") && !dontContinue){
				addToLog("Collecting continuation information", "allPages()", DEBUG);
				notdone = true;
				url = getContinuationInfo(result, url, "apfrom");
				
			}
			addToLog("Gathering page data", "allPages()", DEBUG);
			
			result = trimResult(result, "allpages");
			if(result == null){
				addToLog("Server returned no results!", "allPages()", WARNING);
				return null;
			}
			
			pagedata.addAll(collectData(result, "p"));
			
		}
		
		return pagedata;
	}
	
	public APIDataList allLinks(String from, String to, String prefix, boolean unique, String[] props, 
			Namespace namespace, int limit){
		
		APIDataList linkdata = new APIDataList();
		addToLog("Entering method", "allLinks()", DEBUG);
		String url = "query&list=alllinks";
		if(notEmpty(from)){
			url += "&alfrom=" + encode(from);
		}
		if(notEmpty(to)){
			url += "&alto=" + encode(to);
		}
		if(notEmpty(prefix)){
			url += "&alprefix=" + encode(prefix);
		}
		if(limit > 0){
			url += "&allimit=" + limit;
		}
		if(namespace != null){
			url += "&alnamespace=" + namespace.getNumber();
		}
		if(unique){
			url += "&alunique=true";
		}
		if(props != null){
			String proplist = "";
			for(String prop : props){
				if(prop.equals("title")){
					if(proplist.isEmpty()){
						proplist = "&alprop=";
					}
					else{
						proplist += "|";
					}
					proplist += prop;
				}
				else if(prop.equals("ids") && !unique){
					if(proplist.isEmpty()){
						proplist = "&alprop=";
					}
					else{
						proplist += "|";
					}
					proplist += prop;
				}
				else if(prop.equals("ids")){
					addToLog("The 'ids' property cannot be used in " +
							"conjuction with 'unique'. 'ids' will be ignored.", "allLinks()", WARNING);
				}
			}
			url += proplist;
		}
		
		addToLog("Query construction complete", "allLinks()", DEBUG);
		
		boolean dontContinue = false;
		// If we expect a finite number of users, and the user didn't say how many...
		if(((notEmpty(from) && notEmpty(to)) || notEmpty(prefix) ) && limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		else if(limit <= 0){
			dontContinue = true;
		}
		
		boolean notdone = true;
		while(notdone && (linkdata == null || linkdata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "allLinks()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue") && !dontContinue){
				addToLog("Collecting continuation information", "allLinks()", DEBUG);
				notdone = true;
				url = getContinuationInfo(result, url, "alcontinue");
				
			}
			addToLog("Gathering link data", "allLinks()", DEBUG);
			
			result = trimResult(result, "alllinks");
			if(result == null){
				addToLog("Server returned no results!", "allLinks()", WARNING);
				return null;
			}
			
			linkdata.addAll(collectData(result, "l"));
			
		}
		
		return linkdata;
	}
	
	public APIDataList allCategories(String from, String to, String prefix, boolean descending, 
			String[] props, int limit){
		
		APIDataList catdata = new APIDataList();
		addToLog("Entering method", "allCategories()", DEBUG);
		String url = "query&list=allcategories";
		if(notEmpty(from)){
			url += "&acfrom=" + encode(from);
		}
		if(notEmpty(to)){
			url += "&acto=" + encode(to);
		}
		if(notEmpty(prefix)){
			url += "&acprefix=" + encode(prefix);
		}
		if(limit > 0){
			url += "&aclimit=" + limit;
		}
		if(descending){
			url += "&acdir=descending";
		}
		if(props != null){
			String proplist = "";
			for(String prop : props){
				if(prop.equals("size") || prop.equals("hidden")){
					if(proplist.isEmpty()){
						proplist = "&acprop=";
					}
					else{
						proplist += "|";
					}
					proplist += prop;
				}
			}
			url += proplist;
		}
		
		addToLog("Query construction complete", "allCategories()", DEBUG);
		
		boolean dontContinue = false;
		// If we expect a finite number of users, and the user didn't say how many...
		if(((notEmpty(from) && notEmpty(to)) || notEmpty(prefix) ) && limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		else if(limit <= 0){
			dontContinue = true;
		}
		
		boolean notdone = true;
		while(notdone && (catdata == null || catdata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "allCategories()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue") && !dontContinue){
				addToLog("Collecting continuation information", "allCategories()", DEBUG);
				notdone = true;
				url = getContinuationInfo(result, url, "acfrom");
				
			}
			addToLog("Gathering category data", "allCategories()", DEBUG);
			
			result = trimResult(result, "allcategories");
			if(result == null){
				addToLog("Server returned no results!", "allCategories()", WARNING);
				return null;
			}
			
			catdata.addAll(collectData(result, "c"));
			
		}
		
		return catdata;
	}
	
	public APIDataList backlinks(String title, Namespace[] namespaces, int filterRedirs, int limit, boolean redirect){
		
		APIDataList linkdata = new APIDataList();
		addToLog("Entering method", "backlinks()", DEBUG);
		String url = "query&list=backlinks";
		if(notEmpty(title)){
			url += "&bltitle=" + encode(title);
		}
		else{
			addToLog("'title' parameter is required! Returning null.", "backlinks()", ERROR);
			return null;
		}
		if(limit > 0){
			url += "&bllimit=" + limit;
		}
		if(namespaces != null){
			String nslist = "&blnamespace=";
			for( Namespace namespace : namespaces ){
				if(!nslist.equals("&blnamespace=")){
					nslist += "|";
				}
				nslist += namespace.getNumber();
			}
			url += nslist;
		}
		if(filterRedirs != 0){
			url += "&blfilterredir=" + (filterRedirs > 0 ? "redirects" : "nonredirects");
		}
		if(redirect){
			addToLog("With 'redirect' enabled, you may receive more data than was requested.", "backlinks()", NOTICE);
			url += "&blredirect=true";
		}
		
		addToLog("Query construction complete", "backlinks()", DEBUG);
		
		// If we expect a finite number of users, and the user didn't say how many...
		if(limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		
		boolean notdone = true;
		while(notdone && (linkdata == null || linkdata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "backlinks()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue")){
				addToLog("Collecting continuation information", "backlinks()", DEBUG);
				notdone = true;
				url = getContinuationInfo(result, url, "blcontinue");
				
			}
			addToLog("Gathering link data", "backlinks()", DEBUG);
			
			result = trimResult(result, "backlinks");
			if(result == null){
				addToLog("Server returned no results!", "backlinks()", WARNING);
				return null;
			}
			
			linkdata.addAll(collectData(result, "bl"));
			
		}
		
		return linkdata;
	}
	
	public APIDataList blocks(String start, String end, boolean newer, String[] ids, String[] users,
			String ip, int limit, String[] props){
		
		APIDataList blockdata = new APIDataList();
		addToLog("Entering method", "blocks()", DEBUG);
		String url = "query&list=blocks";
		if(notEmpty(start)){
			if(start.matches("20[0-9]{2,2}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z")){
				url += "&bkstart=" + encode(start);
			}
			else{
				addToLog("Timestamp for 'start' is malformed! Ignoring parameter", "blocks()", WARNING);
			}
		}
		if(notEmpty(end)){
			if(end.matches("20[0-9]{2,2}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z")){
				url += "&bkend=" + encode(end);
			}
			else{
				addToLog("Timestamp for 'end' is malformed! Ignoring parameter", "blocks()", WARNING);
			}
		}
		if(limit > 0){
			url += "&bklimit=" + limit;
		}
		if(newer){
			url += "&bkdir=newer";
		}
		if(props != null){
			String proplist = "";
			for(String prop : props){
				if(prop.equals("id") || prop.equals("user") ||
						prop.equals("by") || prop.equals("timestamp") ||
						prop.equals("expiry") || prop.equals("reason") ||
						prop.equals("range") || prop.equals("flags")){
					if(proplist.isEmpty()){
						proplist = "&bkprop=";
					}
					else{
						proplist += "|";
					}
					proplist += prop;
				}
			}
			url += proplist;
		}
		if(ids != null){
			String idlist = "&bkids=";
			for(String id : ids){
				if(!idlist.equals("&bkids=")){
					idlist += "|";
				}
				idlist += id;
			}
			url += idlist;
		}
		if(users != null){
			if(ids != null){
				addToLog("Using 'users' with 'ids' may cause no results to be returned", "blocks()", NOTICE);
			}
			String userlist = "&bkusers=";
			for(String user : users){
				if(!userlist.equals("&bkusers=")){
					userlist += "|";
				}
				userlist += encode(user);
			}
		}
		if(notEmpty(ip)){
			if(users != null){
				addToLog("'ip' cannot be used in conjunction with 'users'. Ignoring paramter.", 
						"blocks()", WARNING);
			}
			else{
				if(ids != null){
					addToLog("Using 'ip' with 'ids' may cause no results to be returned", 
							"blocks()", NOTICE);
				}
				if(ip.matches("([0-2]?[0-9]{1,2}\\.){3,3}[0-2]?[0-9]{1,2}(/[1-3]?[0-9])?")){
					if(ip.contains("/")){
						int index = ip.indexOf("/");
						int CIDR = Integer.parseInt(ip.substring(index + 1));
						if(CIDR < 16){
							addToLog("Size of IP range exceeds /16. Ignoring parameter.", "blocks()", WARNING);
						}
						else{
							url += "&bkip=" + ip;
						}
					}
					else{
						url += "&bkip=" + ip;
					}
				}
				else{
					addToLog("Value for 'ip' is not a valid IP address or range! Ignoring parameter.", 
							"blocks()", WARNING);
				}
			}
		}
		
		addToLog("Query construction complete", "blocks()", DEBUG);
		
		boolean dontContinue = false;
		// If we expect a finite number of users, and the user didn't say how many...
		if(((notEmpty(start) && notEmpty(end)) || users != null || 
				notEmpty(ip) || ids != null ) && limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		else if(limit <= 0){
			dontContinue = true;
		}
		
		boolean notdone = true;
		while(notdone && (blockdata == null || blockdata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "blocks()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue") && !dontContinue){
				addToLog("Collecting continuation information", "blocks()", DEBUG);
				notdone = true;
				url = getContinuationInfo(result, url, "bkstart");
				
			}
			addToLog("Gathering block data", "blocks()", DEBUG);
			
			result = trimResult(result, "blocks");
			if(result == null){
				addToLog("Server returned no results!", "blocks()", WARNING);
				return null;
			}
			
			blockdata.addAll(collectData(result, "block"));
			
		}
		
		return blockdata;
	}
	
	public APIDataList categoryMembers(String category, Namespace[] namespaces, boolean descending, String sort,
			String[] types, String startTimestamp, String endTimestamp, String startSort, String endSort, 
			String[] props, int limit){
		
		APIDataList catdata = new APIDataList();
		addToLog("Entering method", "categoryMembers()", DEBUG);
		String url = "query&list=categorymembers";
		if(notEmpty(category)){
			if(!category.contains("Category:")){
				category = "Category:" + category;
			}
			url += "&cmtitle=" + encode(category);
		}
		else{
			addToLog("'category' argument is required! Returning null.", "categoryMembers()", ERROR);
		}
		if(notEmpty(sort) && !sort.equals("sortkey")){
			if(sort.equals("timestamp")){
				if(notEmpty(startTimestamp)){
					if(startTimestamp.matches("20[0-9]{2,2}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z")){
						url += "&cmstart=" + encode(startTimestamp);
					}
					else{
						addToLog("Timestamp for 'startTimestamp' is malformed! Ignoring parameter", 
								"categoryMembers()", WARNING);
					}
				}
				if(notEmpty(endTimestamp)){
					if(endTimestamp.matches("20[0-9]{2,2}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z")){
						url += "&cmend=" + encode(endTimestamp);
					}
					else{
						addToLog("Timestamp for 'endTimestamp' is malformed! Ignoring parameter", 
								"categoryMembers()", WARNING);
					}
				}
			}
			else{
				addToLog("Value for 'sort' not recognized. Ignoring.", "categoryMembers()", WARNING);
			}
		}
		// 'sort' is empty or is "sortkey" (default)
		else{
			if(notEmpty(startSort)){
				url += "&cmstartsortkey=" + encode(startSort);
			}
			if(notEmpty(endSort)){
				url += "&cmendsortkey=" + encode(endSort);
			}
			if(types != null){
				String typelist = "";
				for(String type : types){
					if(type.equals("page") || type.equals("subcat") || type.equals("file")){
						if(typelist.isEmpty()){
							typelist = "&cmtype=";
						}
						else{
							typelist += "|";
						}
						typelist += type;
					}
				}
				url += typelist;
			}
		}
		if(limit > 0){
			url += "&cmlimit=" + limit;
		}
		if(descending){
			url += "&cmdir=desc";
		}
		if(namespaces != null){
			String nslist = "&cmnamespace=";
			for(Namespace ns : namespaces){
				if(!nslist.equals("&cmnamespace=")){
					nslist += "|";
				}
				nslist += ns.getNumber();
			}
		}
		if(props != null){
			String proplist = "";
			for(String prop : props){
				if(prop.equals("ids") || prop.equals("title") ||
						prop.equals("sortkey") || prop.equals("timestamp") ||
						prop.equals("sortkeyprefix") || prop.equals("type")){
					if(proplist.isEmpty()){
						proplist = "&cmprop=";
					}
					else{
						proplist += "|";
					}
					proplist += prop;
				}
			}
			url += proplist;
		}

		
		addToLog("Query construction complete", "categoryMembers()", DEBUG);
		
		boolean dontContinue = false;
		// If we expect a finite number of users, and the user didn't say how many...
		if((notEmpty(startTimestamp) && notEmpty(endTimestamp) && notEmpty(sort) && sort.equals("timestamp")) || 
				(notEmpty(startSort) && notEmpty(endSort) && (!notEmpty(sort) || sort.equals("sortkey")))
				&& limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		else if(limit <= 0){
			dontContinue = true;
		}
		
		boolean notdone = true;
		while(notdone && (catdata == null || catdata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "cmcontinue()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue") && !dontContinue){
				addToLog("Collecting continuation information", "categoryMembers()", DEBUG);
				notdone = true;
				url = getContinuationInfo(result, url, "cmcontinue");
				
			}
			addToLog("Gathering category page data", "categoryMembers()", DEBUG);
			
			result = trimResult(result, "categorymembers");
			if(result == null){
				addToLog("Server returned no results!", "categoryMembers()", WARNING);
				return null;
			}
			
			catdata.addAll(collectData(result, "cm"));
			
		}
		
		return catdata;
	}
	
	// TODO: deletedRevisions() (requires administrative or researcher access)
	
	public APIDataList embeddedIn(String title, Namespace[] namespaces, int filterRedirs, int limit){
		
		APIDataList linkdata = new APIDataList();
		addToLog("Entering method", "embeddedIn()", DEBUG);
		String url = "query&list=embeddedin";
		if(notEmpty(title)){
			url += "&eititle=" + encode(title);
		}
		else{
			addToLog("'title' parameter is required! Returning null.", "embeddedIn()", ERROR);
			return null;
		}
		if(limit > 0){
			url += "&eilimit=" + limit;
		}
		if(namespaces != null){
			String nslist = "&einamespace=";
			for( Namespace namespace : namespaces ){
				if(!nslist.equals("&einamespace=")){
					nslist += "|";
				}
				nslist += namespace.getNumber();
			}
			url += nslist;
		}
		if(filterRedirs != 0){
			url += "&eifilterredir=" + (filterRedirs > 0 ? "redirects" : "nonredirects");
		}
		
		addToLog("Query construction complete", "embeddedIn()", DEBUG);
		
		// If we expect a finite number of users, and the user didn't say how many...
		if(limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		
		boolean notdone = true;
		while(notdone && (linkdata == null || linkdata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "embeddedIn()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue")){
				addToLog("Collecting continuation information", "embeddedIn()", DEBUG);
				notdone = true;
				url = getContinuationInfo(result, url, "eicontinue");
				
			}
			addToLog("Gathering transclusion data", "embeddedIn()", DEBUG);
			
			result = trimResult(result, "embeddedin");
			if(result == null){
				addToLog("Server returned no results!", "embeddedIn()", WARNING);
				return null;
			}
			
			linkdata.addAll(collectData(result, "ei"));
			
		}
		
		return linkdata;
	}
	
	// TODO: fileArchive() (requires administrative or researcher rights)
	
	public APIDataList imageUsage(String title, Namespace[] namespaces, int filterRedirs, int limit, boolean redirect){
		
		APIDataList linkdata = new APIDataList();
		addToLog("Entering method", "imageUsage()", DEBUG);
		String url = "query&list=imageusage";
		if(notEmpty(title)){
			if(!title.contains("File:")){
				title = "File:" + title;
			}
			url += "&iutitle=" + encode(title);
		}
		else{
			addToLog("'title' parameter is required! Returning null.", "imageUsage()", ERROR);
			return null;
		}
		if(limit > 0){
			url += "&iulimit=" + limit;
		}
		if(namespaces != null){
			String nslist = "&iunamespace=";
			for( Namespace namespace : namespaces ){
				if(!nslist.equals("&iunamespace=")){
					nslist += "|";
				}
				nslist += namespace.getNumber();
			}
			url += nslist;
		}
		if(filterRedirs != 0){
			url += "&iufilterredir=" + (filterRedirs > 0 ? "redirects" : "nonredirects");
		}
		if(redirect){
			url += "&iuredirect=true";
		}
		
		addToLog("Query construction complete", "imageUsage()", DEBUG);
		
		// If we expect a finite number of users, and the user didn't say how many...
		if(limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		
		boolean notdone = true;
		while(notdone && (linkdata == null || linkdata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "imageUsage()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue")){
				addToLog("Collecting continuation information", "imageUsage()", DEBUG);
				notdone = true;

				url = getContinuationInfo(result, url, "iucontinue");
			}
			addToLog("Gathering transclusion data", "imageUsage()", DEBUG);
			
			result = trimResult(result, "imageusage");
			if(result == null){
				addToLog("Server returned no results!", "imageUsage()", WARNING);
				return null;
			}
			
			linkdata.addAll(collectData(result, "iu"));
			
		}
		
		return linkdata;
	}
	
	public APIDataList interwikiBacklinks(String prefix, String title, String[] props, int limit){
		
		APIDataList linkdata = new APIDataList();
		addToLog("Entering method", "interwikiBacklinks()", DEBUG);
		String url = "query&list=iwbacklinks";
		if(notEmpty(prefix)){
			url += "&iwblprefix=" + encode(prefix);
			if(notEmpty(title)){
				url += "&iwbltitle=" + encode(title);
			}
		}
		else if(notEmpty(title)){
			addToLog("'title' parameter must be used with 'prefix'. Ignoring 'title'.", 
					"interwikiBacklinks()", WARNING);
		}
		if(limit > 0){
			url += "&iwbllimit=" + limit;
		}
		if(props != null){
			String proplist = "&iwblprop=";
			for( String prop : props ){
				if(prop.equals("iwprefix") || prop.equals("iwtitle")){
					if(!proplist.equals("&iwblprop=")){
						proplist += "|";
					}
					proplist += prop;
				}
			}
			url += proplist;
		}
		
		addToLog("Query construction complete", "interwikiBacklinks()", DEBUG);
		
		// If we expect a finite number of users, and the user didn't say how many...
		if(limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		
		boolean notdone = true;
		while(notdone && (linkdata == null || linkdata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "interwikiBacklinks()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue")){
				addToLog("Collecting continuation information", "interwikiBacklinks()", DEBUG);
				notdone = true;

				url = getContinuationInfo(result, url, "iwblcontinue");
			}
			addToLog("Gathering backlink data", "interwikiBacklinks()", DEBUG);
			
			result = trimResult(result, "iwbacklinks");
			if(result == null){
				addToLog("Server returned no results!", "interwikiBacklinks()", WARNING);
				return null;
			}
			
			linkdata.addAll(collectData(result, "iw"));
			
		}
		
		return linkdata;
	}
	
	public APIDataList logEvents(String type, String action, String start, String end, 
			boolean newer, String user, String title, String[] props, int limit){
		
		APIDataList logdata = new APIDataList();
		addToLog("Entering method", "logEvents()", DEBUG);
		String url = "query&list=logevents";
		if(notEmpty(action)){
			// An if statement for all of this would be awful. Whoever wrote this API,
			// and particularly that "review" module, should be shot.
			String[] actions = {"block/block", "block/unblock", "block/reblock", "protect/protect", 
					"protect/modify", "protect/unprotect", "protect/move_prot", "rights/rights", 
					"rights/disable", "delete/delete", "delete/restore", "delete/revision", 
					"delete/event", "upload/upload", "upload/overwrite", "upload/revert", "move/move",
					"move/move_redir", "import/upload", "import/interwiki", "merge/merge", 
					"suppress/revision", "suppress/file", "suppress/event", "suppress/delete", 
					"suppress/block", "suppress/reblock", "patrol/patrol", "review/approve", 
					"review/approve2", "review/approve-i", "review/approve2-i", "review/approve-a", 
					"review/approve2-a", "review/approve-ia", "review/approve2-ia", "review/unapprove", 
					"review/unapprove2", "rights/erevoke", "suppress/setstatus", "suppress/hide-afl", 
					"suppress/unhide-afl", "newusers/newusers", "newusers/create", "newusers/create2", 
					"newusers/autocreate"};
			boolean found = false;
			for(String act : actions){
				if(action.equals(act)){
					url += "&leaction=" + encode(action);
					found = true;
					break;
				}
			}
			if(!found){
				addToLog("Action " + action + " not recognized. Parameter ignored.", "logEvents()", WARNING);
			}

		}
		else if(notEmpty(type)){
			if(type.equals("block") || type.equals("protect") || type.equals("rights") ||
					type.equals("delete") || type.equals("upload") || type.equals("move") ||
					type.equals("import") || type.equals("patrol") || type.equals("merge") ||
					type.equals("suppress") || type.equals("review") || type.equals("stable") ||
					type.equals("renameuser") || type.equals("newusers")){
				url += "&letype=" + encode(type);
			}
			else{
				addToLog("Log type" + type + " not recognized. Parameter ignored.", "logEvents()", WARNING);
			}
		} 
		if(limit > 0){
			url += "&lelimit=" + limit;
		}
		if(props != null){
			String proplist = "&leprop=";
			for( String prop : props ){
				if(prop.equals("ids") || prop.equals("title") || prop.equals("type") || 
						prop.equals("user") || prop.equals("userid") || prop.equals("timestamp") || 
						prop.equals("comment") || prop.equals("parsedcomment") || 
						prop.equals("details") || prop.equals("tags")){
					if(!proplist.equals("&leprop=")){
						proplist += "|";
					}
					proplist += prop;
				}
			}
			url += proplist;
		}
		if(notEmpty(start)){
			if(start.matches("20[0-9]{2,2}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z")){
				url += "&lestart=" + encode(start);
			}
			else{
				addToLog("Timestamp for 'start' is malformed! Ignoring parameter", "logEvents()", WARNING);
			}
		}
		if(notEmpty(end)){
			if(end.matches("20[0-9]{2,2}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z")){
				url += "&leend=" + encode(end);
			}
			else{
				addToLog("Timestamp for 'end' is malformed! Ignoring parameter", "logEvents()", WARNING);
			}
		}
		if(newer){
			url += "&ledir=newer";
		}
		if(notEmpty(user)){
			url += "&leuser=" + encode(user);
		}
		if(notEmpty(title)){
			url += "&letitle=" + encode(title);
		}
		
		addToLog("Query construction complete", "logEvents()", DEBUG);
		
		// If we expect a finite number of users, and the user didn't say how many...
		if(limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		
		boolean notdone = true;
		while(notdone && (logdata == null || logdata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "logEvents()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue")){
				addToLog("Collecting continuation information", "logEvents()", DEBUG);
				notdone = true;

				url = getContinuationInfo(result, url, "lestart");
			}
			addToLog("Gathering log data", "logEvents()", DEBUG);
			
			result = trimResult(result, "logevents");
			if(result == null){
				addToLog("Server returned no results!", "logEvents()", WARNING);
				return null;
			}
			
			logdata.addAll(collectData(result, "item"));
			
		}
		
		return logdata;
	}
	
	// Warning: "patrolled" requires the patrol right for some reason.
	public APIDataList recentChanges(Namespace[] namespaces, String user, boolean exclude, String start, String end, 
			boolean newer, String[] show, String[] types, String[] props, int limit){
		
		APIDataList logdata = new APIDataList();
		addToLog("Entering method", "recentChanges()", DEBUG);
		String url = "query&list=recentchanges";
		if(notEmpty(user)){
			url += (exclude ? "&rcexcludeuser=" : "&rcuser=") + encode(user);
		}
		if(types != null){
			String typelist = "&rctype=";
			for(String type : types){
				if(type.equals("edit") || type.equals("new") || type.equals("log")){
					if(!typelist.equals("&rctype=")){
						typelist += "|";
					}
					typelist += type;
				}
				else{
					addToLog("Log type" + type + " not recognized. Parameter ignored.", "recentChanges()", WARNING);
				}
			}
			
			url += typelist;
		} 
		if(limit > 0){
			url += "&rclimit=" + limit;
		}
		if(props != null){
			String proplist = "&rcprop=";
			for( String prop : props ){
				if(prop.equals("user") || prop.equals("userid") || prop.equals("comment") || 
						prop.equals("parsedcomment") || prop.equals("flags") || prop.equals("timestamp") || 
						prop.equals("title") || prop.equals("ids") || prop.equals("sizes") ||
						 prop.equals("redirect") || prop.equals("patrolled") || prop.equals("loginfo")){
					if(!proplist.equals("&rcprop=")){
						proplist += "|";
					}
					proplist += prop;
				}
			}
			url += proplist;
		}
		if(notEmpty(start)){
			if(start.matches("20[0-9]{2,2}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z")){
				url += "&rcstart=" + encode(start);
			}
			else{
				addToLog("Timestamp for 'start' is malformed! Ignoring parameter", "recentChanges()", WARNING);
			}
		}
		if(notEmpty(end)){
			if(end.matches("20[0-9]{2,2}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z")){
				url += "&rcend=" + encode(end);
			}
			else{
				addToLog("Timestamp for 'end' is malformed! Ignoring parameter", "recentChanges()", WARNING);
			}
		}
		if(newer){
			url += "&rcdir=newer";
		}
		if(namespaces != null){
			String nslist = "&rcnamespace=";
			for(Namespace ns : namespaces){
				if(!nslist.equals("&rcnamespace=")){
					nslist += "|";
				}
				nslist += ns.getNumber();
			}
		}
		if(show != null){
			String showlist = "&rcshow=";
			for(String opt : show){
				if(opt.matches("!?(minor|bot|anon|redirect|patrolled)")){
					if(!showlist.equals("&rcshow=")){
						showlist += "|";
					}
					showlist += opt;
				}
				else{
					addToLog("Unrecognized show value " + opt + " ignored.", "recentChanges()", WARNING);
				}
			}
			url += showlist;
		}
		
		addToLog("Query construction complete", "recentChanges()", DEBUG);
		
		// If we expect a finite number of users, and the user didn't say how many...
		if(limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		
		boolean notdone = true;
		while(notdone && (logdata == null || logdata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "recentChanges()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue")){
				addToLog("Collecting continuation information", "recentChanges()", DEBUG);
				notdone = true;

				url = getContinuationInfo(result, url, "rcstart");
			}
			addToLog("Gathering log data", "recentChanges()", DEBUG);
			
			result = trimResult(result, "recentchanges");
			if(result == null){
				addToLog("Server returned no results!", "recentChanges()", WARNING);
				return null;
			}
			
			logdata.addAll(collectData(result, "rc"));
			
		}
		
		return logdata;
	}
	
	// TODO: Search? Will this even be useful for bots? Thinking no....
	
	// Tags not supported (no abusefilter stuff is)
	
	// Warning: "patrolled" requires the patrol right for some reason.
	public APIDataList userContribs(Namespace[] namespaces, String user, String prefix, String start, String end, 
			boolean newer, String[] show, String[] props, int limit){
		
		APIDataList logdata = new APIDataList();
		addToLog("Entering method", "userContribs()", DEBUG);
		String url = "query&list=usercontribs";
		if(notEmpty(prefix)){
			url += "&ucuserprefix=" + encode(prefix);
		}
		else if(notEmpty(user)){
			url += "&ucuser=" + encode(user);
		}
		else{
			addToLog("Either user or prefix must be specified!", "userContribs()", ERROR);
			return null;
		}
		if(limit > 0){
			url += "&uclimit=" + limit;
		}
		if(props != null){
			String proplist = "&ucprop=";
			for( String prop : props ){
				if(prop.equals("comment") || prop.equals("parsedcomment") || prop.equals("flags") || 
						prop.equals("timestamp") || prop.equals("title") || prop.equals("ids") || 
						prop.equals("size") || prop.equals("patrolled")){
					if(!proplist.equals("&ucprop=")){
						proplist += "|";
					}
					proplist += prop;
				}
			}
			url += proplist;
		}
		if(notEmpty(start)){
			if(start.matches("20[0-9]{2,2}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z")){
				url += "&ucstart=" + encode(start);
			}
			else{
				addToLog("Timestamp for 'start' is malformed! Ignoring parameter", "userContribs()", WARNING);
			}
		}
		if(notEmpty(end)){
			if(end.matches("20[0-9]{2,2}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z")){
				url += "&ucend=" + encode(end);
			}
			else{
				addToLog("Timestamp for 'end' is malformed! Ignoring parameter", "userContribs()", WARNING);
			}
		}
		if(newer){
			url += "&ucdir=newer";
		}
		if(namespaces != null){
			String nslist = "&ucnamespace=";
			for(Namespace ns : namespaces){
				if(!nslist.equals("&ucnamespace=")){
					nslist += "|";
				}
				nslist += ns.getNumber();
			}
		}
		if(show != null){
			String showlist = "&ucshow=";
			for(String opt : show){
				if(opt.matches("!?(minor|patrolled)")){
					if(!showlist.equals("&ucshow=")){
						showlist += "|";
					}
					showlist += opt;
				}
				else{
					addToLog("Unrecognized show value " + opt + " ignored.", "userContribs()", WARNING);
				}
			}
			url += showlist;
		}
		
		addToLog("Query construction complete", "userContribs()", DEBUG);
		
		// If we expect a finite number of users, and the user didn't say how many...
		if(limit <= 0){
			// ...get all of them
			limit = Integer.MAX_VALUE;
		}
		
		boolean notdone = true;
		while(notdone && (logdata == null || logdata.size() < limit)){
			notdone = false;
			String result = doRequest(url, "userContribs()", false);
			if(result == null){
				// IOException occurred, returning null to allow graceful shutdown
				return null;
			}
			// Handle continuation stuff
			if(result.contains("query-continue")){
				addToLog("Collecting continuation information", "userContribs()", DEBUG);
				notdone = true;

				url = getContinuationInfo(result, url, (notEmpty(prefix) ? "uccontinue" : "ucstart"));
			}
			addToLog("Gathering log data", "userContribs()", DEBUG);
			
			result = trimResult(result, "usercontribs");
			if(result == null){
				addToLog("Server returned no results!", "userContribs()", WARNING);
				return null;
			}
			
			logdata.addAll(collectData(result, "item"));
			
		}
		
		return logdata;
	}
	
	private String getContinuationInfo(String result, String url, String continueArg){
		String queryCont = result.substring(result.indexOf("query-continue"));
		String cont = getAttribute(queryCont, continueArg);
		if(url.contains(continueArg)){
			// Remove existing parameter
			int index1 = url.indexOf("&" + continueArg);
			int index2 = url.indexOf("&", index1 + 2);
			if(index2 != -1){
				url = url.substring(0, index1) + url.substring(index2);
			}
			else{
				url = url.substring(0, index1);
			}
		}
		url += "&" + continueArg + "=" + encode(cont);
		return url;
	}
	
	private String trimResult(String result, String queryname){
		int index1 = result.indexOf("<" + queryname + ">") + queryname.length() + 2;
		if(index1 == queryname.length() + 1){
			return null;
		}
		int index2 = result.indexOf("</" + queryname + ">");
		
		result = result.substring(index1, index2);
		return result;
	}
	
	private APIDataList collectData(String tags, String tagname){
		
		APIDataList data = new APIDataList();
		
		while(tags.contains("<")){
			// There are six possible formats the API will use for a single item.
			// Multiple formats may be used in the same query.
			// 1. <item attr="value" ... />
			// 2. <item attr="value" ... >
			// 		 <subtree>
			// 			<s sattr="svalue" ... />
			//          ...
			//  	 </subtree>
			//    </item>
			// 3. <item attr="value" ... >
			//       <subtree>
			//          <s>svalue</s>
			//          ...
			//       </subtree>
			//    </item>
			// 4. <item attr="value" ... >Name</item>
			// 5. <item attr="value" ... >
			//       <s>svalue</s>
			//       ...
			//    </item>
			// 6. <item attr="value" ... >
			//       <s sattr="svalue" ... >
			//       ...
			//    </item>
			
			// Index of the start of the current item (usually 0)
			int startIndex = tags.indexOf("<" + tagname);
			// Index of the start of the next item
			int nextStartIndex = tags.indexOf("<" + tagname, startIndex + 2); 
			if(nextStartIndex == -1){
				nextStartIndex = tags.length();
			}
			
			String tag = tags.substring(startIndex, nextStartIndex);
			tags = tags.substring(nextStartIndex);
		
			APIData itemData = new APIData();
			// If </item> is part of the tag, there is a subtree of some form
			// (formats 2-6) that must be handled
			if(tag.contains("</" + tagname + ">")){
				int endOfMainTag = tag.indexOf(">") + 1;
				int endOfSubtree = tag.indexOf("</" + tagname + ">");

				String subtree = tag.substring(endOfMainTag, endOfSubtree);

				if(!subtree.contains("<")){
					// If no tags inside, this is format 4
					// <item attr="value" ... >Name</item>

					itemData.put("name", subtree);
				}
				else{
					// We know it contains tags

					int firstTagStart = subtree.indexOf("<");
					int firstTagEnd = subtree.indexOf(">");
					String subTreeName = null;
					String subTagContent = subtree.substring(firstTagStart + 1, firstTagEnd);
					if(subtree.contains("</" + subTagContent + ">")){
						String subSubContent = 
							subtree.substring(firstTagEnd + 1, subtree.indexOf("</" + subTagContent + ">"));
						if(subSubContent.contains("<")){
							// There are tags within the tags we just located
							// We are dealing with format 2 or 3 and must strip out the surrounding tags
							subTreeName = subTagContent;
							subtree = subSubContent;
						}
					}

					// Have to locate these again
					firstTagStart = subtree.indexOf("<");
					firstTagEnd = subtree.indexOf(">") + 1;
					String subTag = subtree.substring(firstTagStart, firstTagEnd);
					if(subTag.contains("/")){
						// We are dealing with format 3 or 6, where the subtree contains
						// data of the form <s sattr="svalue" ... />
						// This is best handled through a recursive call
						String subTagName = subTag.substring(subTag.indexOf("<") + 1, subTag.indexOf(" "));
						APIDataList tagData = collectData(subtree, subTagName);
						itemData.put((subTreeName == null ? subTagName : subTreeName), tagData);
					}
					else{
						// We are dealing with format 2 or 5, where the subtree contains
						// data of the form <s>svalue</s>
						// This can be handled here
						String subTagName = subTag.replace("<", "").replace(">", "");
						APIArray subData = new APIArray();
						while(subtree.contains("<")){
							int endOfOpenTag = subtree.indexOf(">") + 1;
							int startOfCloseTag = subtree.indexOf("<", endOfOpenTag);
							String value = subtree.substring(endOfOpenTag, startOfCloseTag);
							subData.add(value);
							subtree = subtree.substring(subtree.indexOf(">", startOfCloseTag) + 1);
						}
						itemData.put((subTreeName == null ? subTagName : subTreeName), subData);
					}
				}
				
				// Trim entire tag down to exclude the subtree
				tag = tag.substring(0, endOfMainTag);
			}
			// Subtree handling complete
			
			while(tag.contains("=")){
				int indexOfSpace = tag.indexOf(" ") + 1;
				int indexOfEquals = tag.indexOf("=");
				int indexOfOpenQ = tag.indexOf("\"") + 1;
				int indexOfCloseQ = tag.indexOf("\"", indexOfOpenQ);
				String attrName = tag.substring(indexOfSpace, indexOfEquals);
				String attrValue = tag.substring(indexOfOpenQ, indexOfCloseQ);
				if(!attrName.equals("xml:space")){
					itemData.put(attrName, attrValue);
				}
				tag = tag.substring(indexOfCloseQ + 1);
			}
			
			data.add(itemData);
		}
		
		return data;
	}
	
	/**
	 * Checks relevant throttles and stops execution temporarily if the program
	 * is moving too quickly
	 * @param post if this is a wiki-modifying action rather than a simple query
	 */
	private void enforceThrottle(boolean post){
		addToLog("Entering method", "enforceThrottle()", DEBUG);
		boolean noticeGiven = false;
		while((System.currentTimeMillis() - (post ? lastPostTime : lastGetTime)) 
				< ((post ? editThrottle : queryThrottle) * 1000)){
			if(!noticeGiven){
				addToLog("Delaying next connection due to throttle", "enforceThrottle()", INFO);
				noticeGiven = true;
			}
			try{
				Thread.sleep(1000);
			}
			catch(InterruptedException e){/* expected */}
		}
		if(post)
			lastPostTime = System.currentTimeMillis();
		else
			lastGetTime = System.currentTimeMillis();
		addToLog("Leaving method", "enforceThrottle()", DEBUG);
	}
	
	/**
	 * Grabs the response from the API and cleans it up for higher methods
	 * @param query the query URL
	 * @param source the calling method
	 * @param post true if the action will modify the wiki
	 * @return A String containing the server's response or null if there was
	 * a network or API error
	 */
	private String doRequest(String query, String source, boolean post){
		String url = QUERY_BASE + query;
		source = "doRequest() from " + source;
		addToLog("Entering method", source, DEBUG);
		
		if((System.currentTimeMillis() - lastLagCheck) > (lagCheckDelay * 1000)){
			checkServerLag();
		}
		
		enforceThrottle(post);
		
		String response = connect(url, source, post);
		
		if(response == null){
			// IOException occurred
			return null;
		}
		
		if(response.contains("<error")){
			String code = getAttribute(response, "code");
			String info = getAttribute(response, "info");
			addToLog("API returned error " + code + ": " + info, source, ERROR);
			return null;
		}
		
		int index1 = response.indexOf("<api>") + 5;
		int index2 = response.indexOf("</api>");
		
		addToLog("Leaving method", source, DEBUG);
		return response.substring(index1, index2);
	}
	
	private boolean notEmpty(String arg1){
		return arg1 != null && !arg1.isEmpty();
	}
	
	private String encode(String arg1){
		try{
			return URLEncoder.encode(arg1, "UTF-8");
		}
		catch(UnsupportedEncodingException e){
			// this won't happen, but in case the code gets mucked with...
			addToLog("UnsupportedEncodingException encountered. Message: " + e.getMessage(), 
					"allUsers()", EMERGENCY);
			System.exit(-4);
			return null;
		}
	}
	
	/**
	 * Grabs an XML-formatted attribute from the given string
	 * @param tag the source string
	 * @param attribute the attribute to recover
	 * @return the value of the attribute, or null if there was no attribute by that name
	 */
	private String getAttribute(String tag, String attribute){
		addToLog("Entering method", "getAttribute()", DEBUG);
		int index1 = tag.indexOf(attribute + "=\"");
		if(index1 == -1){
			addToLog("Unable to find attribute for " + attribute, "getAttribute()", WARNING);
			return null;
		}
		else{
			index1 += attribute.length() + 2;
		}
		int index2 = tag.indexOf("\"", index1);
		addToLog("Leaving method", "getAttribute()", DEBUG);
		return tag.substring(index1, index2);
	}
	
	/**
	 * Connects to the given url and returns the result
	 * @param url the query url
	 * @param source the calling method(s)
	 * @param post if this needs to be a POST request
	 * @return the server's response, or null if there was a network error
	 */
	private String connect(String url, String source, boolean post){
		addToLog("Entering method", "connect() from " + source, DEBUG);
		try{
			addToLog("Connecting to URL: " + url, "connect() from " + source, DEBUG);
			URLConnection conn = new URL(url).openConnection();
			setCookies(conn, post);
			conn.connect();
			BufferedReader reply = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"));
			if(post){
				collectCookies(conn);
			}
			String response = "";
			String line = null;
			while((line = reply.readLine()) != null){
				response += line + "\n";
			}
			reply.close();
			addToLog("Leaving method", "connect() from " + source, DEBUG);
			return response;
		}
		catch(IOException e){
			// ERROR, not EMERGENCY, to allow graceful shutdown
			addToLog("IOException encountered. Message: " + e.getMessage(), 
					"connect() from " + source, ERROR);
			return null;
		}
	}
	
	/**
	 * Sets cookies on the given connection
	 * @param conn the connection
	 * @param post if login cookies need to be provided
	 */
	private void setCookies(URLConnection conn, boolean post){
		addToLog("Entering method", "setCookies()", DEBUG);
		String cookieList = "";
		if(post){
			for (Map.Entry<String, String> entry : sessionCookies.entrySet()){
				cookieList += entry.getKey() + "=" + entry.getValue() + "; ";
			}
		}
		for (Map.Entry<String, String> entry : cookies.entrySet()){
			cookieList += entry.getKey() + "=" + entry.getValue() + "; ";
		}
		
		conn.setRequestProperty("Cookie", cookieList);
		conn.setRequestProperty("Useragent", useragent);
		addToLog("Leaving method", "setCookies()", DEBUG);
	}
	
	/**
	 * Grabs cookies from the connection
	 * @param conn the connection
	 */
	private void collectCookies(URLConnection conn){
		addToLog("Entering method", "collectCookies()", DEBUG);
		// reset the cookie store
        cookies.clear();
        sessionCookies.clear();
        String headerName = null;
        for (int i = 1; (headerName = conn.getHeaderFieldKey(i)) != null; i++){
            if (headerName.equals("Set-Cookie"))
            {
                String cookie = conn.getHeaderField(i);

                // _session cookies are for cookies2, otherwise this causes problems
                if (cookie.contains("_session")){
                	cookie = cookie.substring(0, cookie.indexOf(';'));
                    String name = cookie.substring(0, cookie.indexOf('='));
                    String value = cookie.substring(cookie.indexOf('=') + 1, cookie.length());
                    sessionCookies.put(name, value);
                }
                else{
                	cookie = cookie.substring(0, cookie.indexOf(';'));
                    String name = cookie.substring(0, cookie.indexOf('='));
                    String value = cookie.substring(cookie.indexOf('=') + 1, cookie.length());
                    cookies.put(name, value);
                }
            }
        }
        addToLog("Leaving method", "collectCookies()", DEBUG);
	}

	/**
	 * Asks the API for the highest replication lag. If too high, suspends
	 * execution to allow the databases time to quit screwing around.
	 */
	private void checkServerLag(){
		addToLog("Entering method", "checkServerLag()", DEBUG);
		lastLagCheck = System.currentTimeMillis();
		boolean exceeds = true;
		while(exceeds){
			String url = QUERY_BASE + "query&meta=siteinfo&siprop=dbrepllag";
			String result = connect(url, "checkServerLag()", false);
			if(result == null){
				// IOException occurred, will be caught by the query method
				return;
			}
			int index = result.indexOf("lag=\"") + 5;
			int lag = Integer.parseInt(result.substring(index, result.indexOf("\"", index + 1)));
			addToLog("Current database lag is " + lag + "s.", "checkServerLag()", INFO);
			exceeds = lag > maxLag;
			if(exceeds){
				try{
					addToLog("Current database lag of " + lag + "s exceeds limit of " + 
							maxLag + "s. Sleeping for 20s.", "checkServerLag()", NOTICE);
					Thread.sleep(20000);
				}
				catch(InterruptedException e){
					// Expected
				}
				addToLog("Resuming operation", "checkServerLag()", DEBUG);
			}
		}
		addToLog("Leaving method", "checkServerLag()", DEBUG);
	}
	
	public static void setOutputLevel(int level){
		if(level >= DEBUG && level <= EMERGENCY){
			outputLevel = level;
		}
	}
	
	public static void setErrorLevel(int level){
		if(level >= DEBUG && level <= EMERGENCY){
			errorLevel = level;
		}
	}
	
	/**
	 * Prints the given message to an output stream based on the provided level.
	 * Messages will not be printed unless the level exceeds outputLevel or errorLevel.
	 * Messages exceeding outputLevel will be printed to System.out; those exceeding
	 * outputLevel will be printed to System.err and System.out. It is recommended to
	 * redirect each stream to a separate logging file.
	 * 
	 * @param message The message to be logged
	 * @param source The calling method(s)
	 * @param level The level of the message
	 */
	public static void addToLog(String message, String source, int level){
		if(level >= outputLevel || level >= errorLevel){

			time.setTimeInMillis(System.currentTimeMillis());
			String entry = "" + time.get(Calendar.MONTH);
			entry += "/" + time.get(Calendar.DAY_OF_MONTH);
			entry += "/" + time.get(Calendar.YEAR);
			entry += " @ " + time.get(Calendar.HOUR_OF_DAY);
			entry += ":" + (time.get(Calendar.MINUTE) < 10 ? "0" : "") + time.get(Calendar.MINUTE);
			entry += ":" + (time.get(Calendar.SECOND) < 10 ? "0" : "") + time.get(Calendar.SECOND);
			
			switch(level){
			case DEBUG: entry += " Debugging information "; break;
			case INFO: entry += " Information "; break;
			case NOTICE: entry += " Notice "; break;
			case WARNING: entry += " WARNING "; break;
			case ERROR: entry += " ERROR "; break;
			case EMERGENCY: entry += " FATAL ERROR "; break;
			default: entry += " Log entry ";
			}
			entry += "from " + source + ":\n\t";
			entry += message;
			if(level >= errorLevel){
				System.err.println(entry);
			}
			System.out.println(entry);
		}
	}
	
	/**
	 * Simple wrapper to avoid "unchecked cast" warnings
	 *
	 */
	public static class APIData{
		private HashMap<String, Object> data = new HashMap<String, Object>();
		
		public APIData(){
			
		}
		
		public Object get(String key){
			return data.get(key);
		}
		
		public void put(String key, Object value){
			data.put(key, value);
		}
		
		public Set<String> keySet(){
			return data.keySet();
		}
		
		public int size(){
			return data.size();
		}
	}
	
	public static class APIDataList implements Iterable<APIData>{
		private ArrayList<APIData> list = new ArrayList<APIData>();
		
		public APIDataList(){
			
		}
		
		public void add(APIData data){
			list.add(data);
		}
		
		public APIData get(int index){
			return list.get(index);
		}
		
		public int size(){
			return list.size();
		}
		
		public void addAll(APIDataList adl){
			// Not accessing it properly, shush, I don't care
			list.addAll(adl.list);
		}
		
		public Iterator<APIData> iterator(){
			return list.iterator();
		}
	}
	
	public static class APIArray implements Iterable<String>{
		private ArrayList<String> array = new ArrayList<String>();
		
		public APIArray(){
			
		}
		
		public void add(String data){
			array.add(data);
		}
		
		public String get(int index){
			return array.get(index);
		}
		
		public int size(){
			return array.size();
		}
		
		public Iterator<String> iterator(){
			return array.iterator();
		}
	}
	
	public static enum Namespace{
		MAIN(":", 0),
		TALK("Talk:", 1),
		USER("User:", 2),
		USER_TALK("User talk:", 3),
		PROJECT("Wikipedia:", 4),
		PROJECT_TALK("Wikipedia talk:", 5),
		FILE("File:", 6),
		FILE_TALK("File talk:", 7),
		MEDIAWIKI("Mediawiki:", 8),
		MEDIAWIKI_TALK("Mediawiki talk:", 9),
		TEMPLATE("Template:", 10),
		TEMPLATE_TALK("Template talk:", 11),
		HELP("Help:", 12),
		HELP_TALK("Help talk:", 13),
		CATEGORY("Category:", 14),
		CATEGORY_TALK("Category talk:", 15),
		PORTAL("Portal:", 100),
		PORTAL_TALK("Portal talk:", 101),
		BOOK("Book:", 108),
		BOOK_TALK("Book talk:", 109),
		SPECIAL("Special:", -1),
		MEDIA("Media:", -2);
		
		private String prefix;
		private int number;
		
		private Namespace(String prefix, int number){
			this.prefix = prefix;
			this.number = number;
		}
		
		public String getPrefix(){
			return prefix;
		}
		
		public String getTitle(){
			return prefix.replace(":", "");
		}
		
		public int getNumber(){
			return number;
		}
		
		public static Namespace getNamespace(String name){
			if(!name.contains(":")){
				name += ":";
			}
			for( Namespace ns : Namespace.values() ){
				if(ns.getPrefix().equalsIgnoreCase(name)){
					return ns;
				}
			}
			addToLog("Namespace " + name + " not recognized. Returning null value.", "Namespace.getNamespace()", ERROR);
			return null;
		}
		
		public static Namespace getNamespaceFromPageTitle(String pagetitle){
			if(pagetitle.contains(":")){
				String name = pagetitle.substring(0, pagetitle.indexOf(":") + 1);
				return getNamespace(name);
			}
			else{
				return MAIN;
			}
		}
		
		public static Namespace getNamespaceFromNumber(int number){
			for( Namespace ns : Namespace.values() ){
				if(ns.getNumber() == number){
					return ns;
				}
			}
			addToLog("Namespace number " + number + " not recognized. Returning null value.", 
					"Namespace.getNamespaceFromNumber()", ERROR);
			return null;
		}
	}
		
	public static enum UserGroup{
		FOUNDER("founder", 131072),
		STEWARD("steward", 65536),
		CHECKUSER("checkuser", 32768),
		OVERSIGHT("oversight", 16384),
		BUREAUCRAT("bureaucrat", 8192),
		SYSOP("sysop", 4096),
		BOT("bot", 2048),
		IP_BLOCK_EXEMPT("ipblock-exempt", 1024),
		ABUSEFILTER("abusefilter", 512),
		FILEMOVER("filemover", 256),
		IMPORTER("import", 128),
		TRANSWIKI("transwiki", 64),
		AUTOREVIEWER("autoreviewer", 32),
		REVIEWER("reviewer", 16),
		ACCOUNT_CREATOR("accountcreator", 8),
		ROLLBACKER("rollbacker", 4),
		RESEARCHER("researcher", 2),
		CONFIRMED("confirmed", 1);
		
		private String name;
		private int level;
		
		private UserGroup(String name, int level){
			this.name = name;
			this.level = level;
		}
		
		public String getName(){
			return name;
		}
		
		public int getLevel(){
			return level;
		}
		
		/**
		 * Returns the type associated with the given name.
		 * Note that this method accepts some colloquial terms. For example,
		 * both "Admin" and "Administrator" will return the SYSOP UserGroup
		 * type (whose actual name is "sysop"). This method is not case sensitive
		 * and any whitespace will be removed and ignored.
		 * 
		 * <p>Please note also that the UserGroup type only contains the groups
		 * available on the English Wikipedia. Those wishing to use this framework
		 * on other wikis may need to update this class for their own needs.
		 * @param name the name of the desired group
		 * @return a UserGroup object, or null if none by that name exists.
		 */
		public static UserGroup getType(String name){
			// Take care of common alternate names
			name = name.replaceAll("[_\\s]", "");
			if(name.equalsIgnoreCase("Admin")){
				return SYSOP;
			}
			if(name.equalsIgnoreCase("Administrator")){
				return SYSOP;
			}
			if(name.equalsIgnoreCase("IPBE")){
				return IP_BLOCK_EXEMPT;
			}
			if(name.equalsIgnoreCase("IPBlockexempt")){
				return IP_BLOCK_EXEMPT;
			}
			if(name.equalsIgnoreCase("Editfilter")){
				return ABUSEFILTER;
			}
			for(UserGroup type : UserGroup.values()){
				if(type.getName().equalsIgnoreCase(name)){
					return type;
				}
			}
			addToLog("Group type " + name + " not recognized. Returning null value.", "UserGroup.getType()", ERROR);
			return null;
		}
	}
	
	public static void main(String[] args){
		NewWiki wiki = new NewWiki("en.wikipedia.org");
		
		String[] props = {"comment", "flags", "timestamp", "title", "ids", "size"};
		APIDataList datalist = wiki.userContribs(null, "YurikBot", "217.121.114.", null, null, false, null, props, 10);
		
		if(datalist == null){
			System.out.println("No results.");
			System.exit(1);
		}
		
		for( APIData entry : datalist ){
			for( String key : entry.keySet() ){
				Object data = entry.get(key);
				System.out.print(key + ": ");
				if(data instanceof String){
					System.out.println((String) data);
				}
				else if(data instanceof APIArray){
					System.out.println();
					for( String datum : (APIArray) data ){
						System.out.println("\t" + datum);
					}
				}
				else if(data instanceof APIDataList){
					System.out.println();
					for( APIData datum : (APIDataList) data ){
						for( String datumKey : datum.keySet() ){
							System.out.println("\t" + datumKey + ": " + datum.get(datumKey));
						}
					}
				}
			}
			System.out.println("\n");
		}
	}
}
